小文件去重合并

set hive.exec.dynamic.partition=true;  
set hive.exec.dynamic.partition.mode=nonstrick;
set mapred.reduce.tasks = 5;
WITH temptable as ( 
  SELECT *,
    ROW_NUMBER() OVER (PARTITION BY dt,request_id, request_method,request_uri,remote_addr
                       ORDER BY `timestamp` DESC) as row_num
  FROM cps.ods_cps_request where dt>'2024-05-30'
)
INSERT OVERWRITE TABLE cps.ods_cps_request partition(dt) 
SELECT 
`request_id`,
`request_method`,
`geoip_city_country_code`,
`realip_remote_addr`,
`geoip_area_code`,
`request_body`,
`request_time`,
`request_uri`,
`bytes_sent`,
`content_type`,
`cookie`,
`cookie_utdma`,
`host`,
`http_host`,
`http_referer`,
`http_user_agent`,
`http_x_forwarded_for`,
`proxy_host`,
`proxy_add_x_forwarded_for`,
`query_string`,
`remote_addr`,
`scheme`,
`status`,
`time`,
`time_local`,
`timestamp`,
`data_time`,
`dt`
FROM temptable
WHERE row_num = 1;
小文件去重合并​

小文件去重合并
